References¶
Dynamic Routing Between Capsules, Sara Sabour et al. https://arxiv.org/pdf/1710.09829.pdf¶
Capsule Networks (CapsNets), Aurélien Geron https://github.com/ageron/handson-ml/blob/master/extra_capsnets.ipynb¶
CapsuleNet on MNIST, Kevin Mader https://www.kaggle.com/kmader/capsulenet-on-mnist¶
capsule-GAN, Guseyn Gadirov https://github.com/gusgad/capsule-GAN/blob/master/capsule_gan.ipynb¶
In [1]:
import numpy as np
import os
import pandas as pd
from keras.preprocessing.image import ImageDataGenerator
from keras import callbacks
from keras.utils.vis_utils import plot_model
import matplotlib.pyplot as plt
import tensorflow as tf
from tensorflow import Graph, Session
import keras.backend as K
from keras import backend as K
from keras import Sequential, models, layers
import keras
from keras.models import Model,Sequential
from keras.datasets import mnist
from keras.layers.advanced_activations import LeakyReLU
from keras.optimizers import adam
from keras.layers import Dense, Activation, Dropout, Convolution2D, Flatten, MaxPooling2D, Reshape, InputLayer, Input
from keras.utils import to_categorical
from keras.layers import Conv2D, MaxPooling2D, Conv2DTranspose
from keras import initializers
from keras.layers import BatchNormalization, Activation, ZeroPadding2D, Lambda, Concatenate, Multiply
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.convolutional import UpSampling2D, Conv2D
from keras.optimizers import Adam
%matplotlib inline
Squash helper function¶
In [0]:
def squash(vector, axis=-1, epsilon=1e-7):
s_squared_norm = K.sum(K.square(vector), axis, keepdims=True)
safe_norm = K.sqrt(s_squared_norm + epsilon)
squash_factor = s_squared_norm / (1.0 + s_squared_norm)
unit_vector = vector / safe_norm
return squash_factor * unit_vector
Primary Capsules Layer¶
In [0]:
def primaryCaps(inputs, dim_vector, n_channels, kernel_size, strides, padding):
'''
inputs : output of first convolutional layer of shape (?x20x20x256)
dim_vector : the dimension of the output vector of each capsule = 8
n_channels : the number of different capsules = 32
output : output of the convultional layer reshaped to ?x1152x8 and squash function applied to it
'''
conv2 = Conv2D(filters=dim_vector*n_channels, kernel_size=kernel_size, strides=strides, padding=padding)(inputs)
out = layers.Reshape(target_shape=[-1, dim_vector])(conv2)
return layers.Lambda(squash)(out)
In [0]:
def build_capsNet():
#input shape of the mnist image
input_shape=(28,28,1)
#this capsnet only has 1 class as it only needs to distinguish if the image is real or fake
num_classes=1
#the image input to the model
x_in = layers.Input(shape=input_shape)
#first concolutional layer
conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', name='conv1')(x_in)
conv1 = LeakyReLU(alpha=0.1)(conv1)
conv1 = BatchNormalization(momentum=0.8)(conv1)
# primary capsules layer
prim = primaryCaps(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
prim = BatchNormalization(momentum=0.8)(prim)
# Digit capsules layer implemented as normal dense layers
x = Flatten()(prim)
uhat = Dense(160, kernel_initializer='he_normal', bias_initializer='zeros', name='uhat_digitcaps')(x)
c = Activation('softmax', name='softmax_digitcaps1')(uhat)
c = Dense(160)(c)
x = Multiply()([uhat, c])
s_j = LeakyReLU()(x)
c = Activation('softmax', name='softmax_digitcaps2')(s_j)
c = Dense(160)(c)
x = Multiply()([uhat, c])
s_j = LeakyReLU()(x)
c = Activation('softmax', name='softmax_digitcaps3')(s_j)
c = Dense(160)(c)
x = Multiply()([uhat, c])
s_j = LeakyReLU()(x)
pred = Dense(1, activation='sigmoid')(s_j)
model = models.Model(x_in , pred)
#model.compile(loss=margin_loss, optimizer=Adam(0.0002, 0.5),metrics={'pred': 'accuracy'})
model.compile(loss='binary_crossentropy', optimizer=Adam(0.0002, 0.5), metrics=['accuracy'])
return model
In [0]:
def margin_loss(y_true, y_pred):
L = y_true * K.square(K.maximum(0., 0.9 - y_pred)) + \
0.5 * (1 - y_true) * K.square(K.maximum(0., y_pred - 0.1))
return K.mean(K.sum(L, 1))
In [6]:
# Define the model
discriminator = build_capsNet()
discriminator.summary()
Load the mnist dataset¶
In [7]:
from keras.datasets import mnist
def load_data():
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = (x_train.astype(np.float32) - 127.5)/127.5
x_test = (x_test.astype(np.float32) - 127.5)/127.5
x_train = x_train.reshape(-1,28,28,1)
x_test = x_test.reshape(-1,28,28,1)
y_train = to_categorical(y_train.astype('float32'))
y_test = to_categorical(y_test.astype('float32'))
return (x_train, y_train, x_test, y_test)
(x_train, y_train,x_test, y_test)=load_data()
print(x_train.shape, y_train.shape, x_test.shape, y_test.shape)
In [0]:
def build_generator():
noise_shape = (100,)
x_noise = Input(shape=noise_shape)
x = Dense(128*7*7, activation="relu")(x_noise)
x = Reshape((7, 7, 128))(x)
x = UpSampling2D()(x)
x = Conv2D(128, kernel_size=3, padding="same")(x)
x = BatchNormalization(momentum=0.8)(x)
x = Activation("relu")(x)
x = UpSampling2D()(x)
x = Conv2D(64, kernel_size=3, padding="same")(x)
x = BatchNormalization(momentum=0.8)(x)
x = Activation("relu")(x)
x = Conv2D(1, kernel_size=3, padding="same")(x)
x_out = Activation('tanh')(x)
model = models.Model(x_noise, x_out)
model.compile(loss='binary_crossentropy', optimizer=Adam(0.0002, 0.5))
return model
In [9]:
generator = build_generator()
generator.summary()
Function to plot the generated images¶
In [0]:
def plot_generated_images(epoch, generator, examples=100, dim=(10,10), figsize=(10,10)):
noise= np.random.normal(loc=0, scale=1, size=[examples, 100])
generated_images = generator.predict(noise)
generated_images = generated_images.reshape(100,28,28)
plt.figure(figsize=figsize)
for i in range(generated_images.shape[0]):
plt.subplot(dim[0], dim[1], i+1)
plt.imshow(generated_images[i], interpolation='nearest')
plt.axis('off')
plt.tight_layout()
plt.savefig('gan_generated_image %d.png' %epoch)
Defining the Adversarial model¶
In [0]:
def create_gan(g, d):
z = Input(shape=[100])
img = g(z)
v = d(g(z))
gan = models.Model(z, v)
gan.compile(loss='binary_crossentropy', optimizer=Adam(0.0002, 0.5))
return gan
In [12]:
gan = create_gan(generator, discriminator)
gan.summary()
Training the model¶
In [0]:
def train(data, epochs, batch_size=128):
#unpack data
x_train = data
K.clear_session()
gen = build_generator()
disc = build_capsNet()
gan = create_gan(gen, disc)
valid = np.ones((batch_size, 1))
fake = np.zeros((batch_size, 1))
for e in range(epochs+1):
real_indxs = np.random.randint(0, x_train.shape[0], batch_size)
real_imgs = x_train[real_indxs]
noise = np.random.normal(0,1, [batch_size, 100])
gen_imgs = gen.predict(noise)
x = np.concatenate([real_imgs, gen_imgs])
y = np.concatenate([valid, fake])
#Training the Discriminator
disc.Trainable = True
d_loss_real = disc.train_on_batch(real_imgs, valid*0.9) # 0.9 for label smoothing
d_loss_fake = disc.train_on_batch(gen_imgs, fake)
d_loss = 0.5 * np.add(d_loss_real, d_loss_fake)
#Training the generator
disc.Trainable = False
noise = np.random.normal(0,1, [batch_size, 100])
y_gan = np.ones((batch_size, 1))
g_loss = gan.train_on_batch((noise), [y_gan])
if e%100 == 0:
#print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (e, d_loss, 100*d_loss, g_loss))
print ("%d [D loss: %f, acc.: %.2f%%] [G loss: %f]" % (e, d_loss[0], 100*d_loss[1], g_loss))
if e % 1000 == 0:
plot_generated_images(e, gen)
In [14]:
train(x_train, epochs=30000)
